package com.kuang.utils;

import com.kuang.pojo.Content;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * @program: kuangshen-es-api
 * @description: 反反爬虫,解析京东页面工具类
 * @author: 陈恩涛
 * @create: 2021-05-26 17:18
 **/

@Component
public class HtmlParseUtil {

    private static PoolingHttpClientConnectionManager clientConnectionManager;

    static {
        // 创建连接池
        clientConnectionManager = new PoolingHttpClientConnectionManager();
        // 最大连接数
        clientConnectionManager.setMaxTotal(100);
        // 设置每个主机最大连接数（比如100个连接，不能都去爬百度，应该分配一些爬虫给知乎）
        clientConnectionManager.setDefaultMaxPerRoute(10);
    }
    public static void main(String[] args) throws IOException {
        //获得请求  https://search.jd.com/Search?keyword=java&enc=utf-8&wq=java&pvid=1d67dd6be9ec494a9035a0be4ec39b62
        // 前提，需要联网！无法获取ajax响应
        new HtmlParseUtil().parseJD("心理学").forEach(System.out::println);
    }

    /**
     * Request配置
     *
     * @return
     */
    private static RequestConfig getRequestConfig() {
        RequestConfig config = RequestConfig
                .custom()
                .setConnectTimeout(1000) // 创建连接的最长时间，毫秒
                .setConnectionRequestTimeout(500) // 获取连接的最长时间，毫秒
                .setSocketTimeout(1000 * 10) // 数据传输的最长时间，毫秒
                .build();
        return config;
    }

    public List<Content> parseJD(String keywords) {
        String url = "https://search.jd.com/Search?keyword="+keywords;

        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(clientConnectionManager).build();
        // 2.添加url，构建HttpGet对象
        HttpGet httpGet = new HttpGet(url);
        httpGet.addHeader("User-Agent",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36");
        // Request配置（比如请求超时如何处理。连接数有限，不可能无节制等待）
        httpGet.setConfig(getRequestConfig());

        CloseableHttpResponse response = null;

        List<Content> goodsList = new ArrayList<>();
        try {
            // 3.发送请求得到响应
            response = httpClient.execute(httpGet);
            // 4.解析
            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                HttpEntity entity = response.getEntity();
                // 响应体不为空
                if (entity != null) {
                    String content = EntityUtils.toString(entity);
                    Document document = Jsoup.parse(content);
                    Element element = document.getElementById("J_goodsList");
                    //获取所有的li元素
                    Elements elements = element.getElementsByTag("li");

                    //获取元素元素中的内容
                    for (Element el : elements) {
                        // 关于这种图片特别多的网站，所有的图片都是延迟加载的
                        // source-data-lazy-img
                        String img = el.getElementsByTag("img").eq(0).attr("data-lazy-img");
                        String price = el.getElementsByClass("p-price").eq(0).text();
                        String title = el.getElementsByClass("p-name").eq(0).text();
                        goodsList.add(new Content().setTitle(title).setImg(img).setPrice(price));
                    }
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                // httpClient不用关闭，连接池会管理
            }
        }
        return goodsList;
    }
}
